<!DOCTYPE HTML>
<html lang="en" >
    <!-- Start book Python爬虫课程讲义 -->
    <head>
        <!-- head:start -->
        <meta charset="UTF-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <title>IT桔子分布式项目2 | Python爬虫课程讲义</title>
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <meta name="description" content="">
        <meta name="generator" content="GitBook 2.6.7">
        <meta name="author" content="BigCat">
        
        <meta name="HandheldFriendly" content="true"/>
        <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
        <meta name="apple-mobile-web-app-capable" content="yes">
        <meta name="apple-mobile-web-app-status-bar-style" content="black">
        <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../../gitbook/images/apple-touch-icon-precomposed-152.png">
        <link rel="shortcut icon" href="../../gitbook/images/favicon.ico" type="image/x-icon">
        
    <link rel="stylesheet" href="../../gitbook/style.css">
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-tbfed-pagefooter/footer.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-splitter/splitter.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-highlight/website.css">
        
    
        
        <link rel="stylesheet" href="../../gitbook/plugins/gitbook-plugin-fontsettings/website.css">
        
    
    

        
    
    
    <link rel="next" href="../../file/duanzi/duanzi.html" />
    
    
    <link rel="prev" href="../../file/part07/7.7.html" />
    

        <!-- head:end -->
    </head>
    <body>
        <!-- body:start -->
        
    <div class="book"
        data-level="7.8"
        data-chapter-title="IT桔子分布式项目2"
        data-filepath="file/part07/7.8.md"
        data-basepath="../.."
        data-revision="Thu Feb 09 2017 09:48:59 GMT+0800 (CST)"
        data-innerlanguage="">
    

<div class="book-summary">
    <nav role="navigation">
        <ul class="summary">
            
            
            
            

            

            
    
        <li class="chapter " data-level="0" data-path="index.html">
            
                
                    <a href="../../index.html">
                
                        <i class="fa fa-check"></i>
                        
                        传智播客Python学院爬虫课程
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1" data-path="file/part01/1.html">
            
                
                    <a href="../../file/part01/1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.</b>
                        
                        爬虫原理与数据抓取
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.1" data-path="file/part01/1.1.html">
            
                
                    <a href="../../file/part01/1.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.1.</b>
                        
                        (了解)通用爬虫和聚焦爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="file/part01/1.2.html">
            
                
                    <a href="../../file/part01/1.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.2.</b>
                        
                        (复习)HTTP/HTTPS的请求与响应
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="file/part01/1.3.html">
            
                
                    <a href="../../file/part01/1.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.3.</b>
                        
                        HTTP/HTTPS抓包工具-Fiddler
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="file/part01/1.4.html">
            
                
                    <a href="../../file/part01/1.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.4.</b>
                        
                        urllib2模块的基本使用
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.5" data-path="file/part01/1.5.html">
            
                
                    <a href="../../file/part01/1.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.5.</b>
                        
                        urllib2：GET请求和POST请求
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.6" data-path="file/part01/1.6.html">
            
                
                    <a href="../../file/part01/1.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.6.</b>
                        
                        urllib2：Handler处理器和自定义Opener
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.7" data-path="file/part01/1.7.html">
            
                
                    <a href="../../file/part01/1.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.7.</b>
                        
                        urllib2：URLError与HTTPError
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="1.8" data-path="file/part01/1.8.html">
            
                
                    <a href="../../file/part01/1.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>1.8.</b>
                        
                        Requests模块
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="2" data-path="file/part02/2.html">
            
                
                    <a href="../../file/part02/2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.</b>
                        
                        非结构化数据与结构化数据提取
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="2.1" data-path="file/part02/2.1.html">
            
                
                    <a href="../../file/part02/2.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.1.</b>
                        
                        正则表达式re模块
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.2" data-path="file/part02/2.2.html">
            
                
                    <a href="../../file/part02/2.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.2.</b>
                        
                        案例：使用正则表达式的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.3" data-path="file/part02/2.3.html">
            
                
                    <a href="../../file/part02/2.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.3.</b>
                        
                        XPath与lxml类库
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.4" data-path="file/part02/2.4.html">
            
                
                    <a href="../../file/part02/2.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.4.</b>
                        
                        案例：使用XPath的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.5" data-path="file/part02/2.5.html">
            
                
                    <a href="../../file/part02/2.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.5.</b>
                        
                        BeautifulSoup4 解析器
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.6" data-path="file/part02/2.6.html">
            
                
                    <a href="../../file/part02/2.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.6.</b>
                        
                        案例：使用bs4的爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.7" data-path="file/part02/2.7.html">
            
                
                    <a href="../../file/part02/2.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.7.</b>
                        
                        JSON模块与JsonPath
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.8" data-path="file/part02/2.8.html">
            
                
                    <a href="../../file/part02/2.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.8.</b>
                        
                        糗事百科案例
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="2.9" data-path="file/part02/2.9.html">
            
                
                    <a href="../../file/part02/2.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>2.9.</b>
                        
                        多线程爬虫案例
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="3" data-path="file/part03/3.html">
            
                
                    <a href="../../file/part03/3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.</b>
                        
                        动态HTML处理和机器图像识别
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="3.1" data-path="file/part03/3.1.html">
            
                
                    <a href="../../file/part03/3.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.1.</b>
                        
                        动态HTML介绍
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.2" data-path="file/part03/3.2.html">
            
                
                    <a href="../../file/part03/3.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.2.</b>
                        
                        Selenium与PhantomJS
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.3" data-path="file/part03/3.3.html">
            
                
                    <a href="../../file/part03/3.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.3.</b>
                        
                        案例一：网站模拟登录
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.4" data-path="file/part03/3.4.html">
            
                
                    <a href="../../file/part03/3.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.4.</b>
                        
                        案例二：动态页面模拟点击
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.5" data-path="file/part03/3.5.html">
            
                
                    <a href="../../file/part03/3.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.5.</b>
                        
                        案例三：执行JavaScript语句
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.6" data-path="file/part03/3.6.html">
            
                
                    <a href="../../file/part03/3.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.6.</b>
                        
                        机器视觉与Tesseract介绍
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.7" data-path="file/part03/3.7.html">
            
                
                    <a href="../../file/part03/3.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.7.</b>
                        
                        处理一些格式规范的文字
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.8" data-path="file/part03/3.8.html">
            
                
                    <a href="../../file/part03/3.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.8.</b>
                        
                        案例：尝试对验证码进行机器识别处理
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="3.9" data-path="file/part03/3.9.html">
            
                
                    <a href="../../file/part03/3.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>3.9.</b>
                        
                        机器学习：训练Tesseract
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="4" data-path="file/part04/4.html">
            
                
                    <a href="../../file/part04/4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.</b>
                        
                        Scrapy框架
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="4.1" data-path="file/part04/4.1.html">
            
                
                    <a href="../../file/part04/4.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.1.</b>
                        
                        配置安装
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.2" data-path="file/part04/4.2.html">
            
                
                    <a href="../../file/part04/4.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.2.</b>
                        
                        入门案例
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.3" data-path="file/part04/4.3.html">
            
                
                    <a href="../../file/part04/4.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.3.</b>
                        
                        Scrapy Shell
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.4" data-path="file/part04/4.4.html">
            
                
                    <a href="../../file/part04/4.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.4.</b>
                        
                        Item Pipeline
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.5" data-path="file/part04/4.5.html">
            
                
                    <a href="../../file/part04/4.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.5.</b>
                        
                        Spiders
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.6" data-path="file/part04/4.6.html">
            
                
                    <a href="../../file/part04/4.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.6.</b>
                        
                        CrawlSpiders
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.7" data-path="file/part04/4.7.html">
            
                
                    <a href="../../file/part04/4.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.7.</b>
                        
                        Request/Response
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.8" data-path="file/part04/4.8.html">
            
                
                    <a href="../../file/part04/4.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.8.</b>
                        
                        Downloader Middlewares
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="4.9" data-path="file/part04/4.9.html">
            
                
                    <a href="../../file/part04/4.9.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>4.9.</b>
                        
                        Settings
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="5" data-path="file/part05/5.html">
            
                
                    <a href="../../file/part05/5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.</b>
                        
                        Scrapy实战项目
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="5.1" data-path="file/part05/5.1.html">
            
                
                    <a href="../../file/part05/5.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.1.</b>
                        
                        (案例一)手机App抓包爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.2" data-path="file/part05/5.2.html">
            
                
                    <a href="../../file/part05/5.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.2.</b>
                        
                        (案例二)阳光热线问政平台爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.3" data-path="file/part05/5.3.html">
            
                
                    <a href="../../file/part05/5.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.3.</b>
                        
                        (案例三)新浪网分类资讯爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.4" data-path="file/part05/5.4.html">
            
                
                    <a href="../../file/part05/5.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.4.</b>
                        
                        (案例四)图片下载器爬虫
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.5" data-path="file/part05/5.5.html">
            
                
                    <a href="../../file/part05/5.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.5.</b>
                        
                        (案例五)将数据保存在MongoDB中
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.6" data-path="file/part05/5.6.html">
            
                
                    <a href="../../file/part05/5.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.6.</b>
                        
                        (案例六)三种scrapy模拟登陆策略
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="5.7" data-path="file/part05/5.7.html">
            
                
                    <a href="../../file/part05/5.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>5.7.</b>
                        
                        附：通过Fiddler进行手机抓包方法
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="6" data-path="file/part06/6.html">
            
                
                    <a href="../../file/part06/6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.</b>
                        
                        scrapy-redis分布式组件
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="6.1" data-path="file/part06/6.1.html">
            
                
                    <a href="../../file/part06/6.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.1.</b>
                        
                        源码分析参考：Connection
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.2" data-path="file/part06/6.2.html">
            
                
                    <a href="../../file/part06/6.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.2.</b>
                        
                        源码分析参考：Dupefilter
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.3" data-path="file/part06/6.3.html">
            
                
                    <a href="../../file/part06/6.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.3.</b>
                        
                        源码分析参考：Picklecompat
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.4" data-path="file/part06/6.4.html">
            
                
                    <a href="../../file/part06/6.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.4.</b>
                        
                        源码分析参考：Pipelines
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.5" data-path="file/part06/6.5.html">
            
                
                    <a href="../../file/part06/6.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.5.</b>
                        
                        源码分析参考：Queue
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.6" data-path="file/part06/6.6.html">
            
                
                    <a href="../../file/part06/6.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.6.</b>
                        
                        源码分析参考：Scheduler
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="6.7" data-path="file/part06/6.7.html">
            
                
                    <a href="../../file/part06/6.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>6.7.</b>
                        
                        源码分析参考：Spider
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="7" data-path="file/part07/7.html">
            
                
                    <a href="../../file/part07/7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.</b>
                        
                        scrapy-redis实战
                    </a>
            
            
            <ul class="articles">
                
    
        <li class="chapter " data-level="7.1" data-path="file/part07/7.1.html">
            
                
                    <a href="../../file/part07/7.1.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.1.</b>
                        
                        源码自带项目说明
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.2" data-path="file/part07/7.2.html">
            
                
                    <a href="../../file/part07/7.2.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.2.</b>
                        
                        有缘网分布式爬虫项目1
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.3" data-path="file/part07/7.3.html">
            
                
                    <a href="../../file/part07/7.3.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.3.</b>
                        
                        有缘网分布式爬虫项目2
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.4" data-path="file/part07/7.4.html">
            
                
                    <a href="../../file/part07/7.4.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.4.</b>
                        
                        处理Redis里的数据
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.5" data-path="file/part07/7.5.html">
            
                
                    <a href="../../file/part07/7.5.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.5.</b>
                        
                        尝试改写新浪网分类资讯爬虫1
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.6" data-path="file/part07/7.6.html">
            
                
                    <a href="../../file/part07/7.6.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.6.</b>
                        
                        尝试改写新浪网分类资讯爬虫2
                    </a>
            
            
        </li>
    
        <li class="chapter " data-level="7.7" data-path="file/part07/7.7.html">
            
                
                    <a href="../../file/part07/7.7.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.7.</b>
                        
                        IT桔子分布式项目1
                    </a>
            
            
        </li>
    
        <li class="chapter active" data-level="7.8" data-path="file/part07/7.8.html">
            
                
                    <a href="../../file/part07/7.8.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>7.8.</b>
                        
                        IT桔子分布式项目2
                    </a>
            
            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="8" data-path="file/duanzi/duanzi.html">
            
                
                    <a href="../../file/duanzi/duanzi.html">
                
                        <i class="fa fa-check"></i>
                        
                            <b>8.</b>
                        
                        课余段子
                    </a>
            
            
        </li>
    


            
            <li class="divider"></li>
            <li>
                <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
                    Published with GitBook
                </a>
            </li>
            
        </ul>
    </nav>
</div>

    <div class="book-body">
        <div class="body-inner">
            <div class="book-header" role="navigation">
    <!-- Actions Left -->
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href="../../" >Python爬虫课程讲义</a>
    </h1>
</div>

            <div class="page-wrapper" tabindex="-1" role="main">
                <div class="page-inner">
                
                
                    <section class="normal" id="section-">
                    
                        <h2 id="&#x9879;&#x76EE;&#x5B9E;&#x73B0;&#xFF1A;">&#x9879;&#x76EE;&#x5B9E;&#x73B0;&#xFF1A;</h2>
<h2 id="itemspy">items.py</h2>
<pre><code class="lang-python">
<span class="hljs-comment"># items.py</span>

<span class="hljs-comment"># -*- coding: utf-8 -*-</span>
<span class="hljs-keyword">import</span> scrapy

<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">CompanyItem</span><span class="hljs-params">(scrapy.Item)</span>:</span>

    <span class="hljs-comment"># &#x516C;&#x53F8;id (url&#x6570;&#x5B57;&#x90E8;&#x5206;)</span>
    info_id = scrapy.Field()
    <span class="hljs-comment"># &#x516C;&#x53F8;&#x540D;&#x79F0;</span>
    company_name = scrapy.Field()
    <span class="hljs-comment"># &#x516C;&#x53F8;&#x53E3;&#x53F7;</span>
    slogan = scrapy.Field()
    <span class="hljs-comment"># &#x5206;&#x7C7B;</span>
    scope = scrapy.Field()
    <span class="hljs-comment"># &#x5B50;&#x5206;&#x7C7B;</span>
    sub_scope = scrapy.Field()

    <span class="hljs-comment"># &#x6240;&#x5728;&#x57CE;&#x5E02;</span>
    city = scrapy.Field()
    <span class="hljs-comment"># &#x6240;&#x5728;&#x533A;&#x57DF;</span>
    area = scrapy.Field()
    <span class="hljs-comment"># &#x516C;&#x53F8;&#x4E3B;&#x9875;</span>
    home_page = scrapy.Field()
    <span class="hljs-comment"># &#x516C;&#x53F8;&#x6807;&#x7B7E;</span>
    tags = scrapy.Field()

    <span class="hljs-comment"># &#x516C;&#x53F8;&#x7B80;&#x4ECB;</span>
    company_intro = scrapy.Field()
    <span class="hljs-comment"># &#x516C;&#x53F8;&#x5168;&#x79F0;&#xFF1A;</span>
    company_full_name = scrapy.Field()
    <span class="hljs-comment"># &#x6210;&#x7ACB;&#x65F6;&#x95F4;&#xFF1A;</span>
    found_time = scrapy.Field()
    <span class="hljs-comment"># &#x516C;&#x53F8;&#x89C4;&#x6A21;&#xFF1A;</span>
    company_size = scrapy.Field()
    <span class="hljs-comment"># &#x8FD0;&#x8425;&#x72B6;&#x6001;</span>
    company_status = scrapy.Field()

    <span class="hljs-comment"># &#x6295;&#x8D44;&#x60C5;&#x51B5;&#x5217;&#x8868;&#xFF1A;&#x5305;&#x542B;&#x83B7;&#x6295;&#x65F6;&#x95F4;&#x3001;&#x878D;&#x8D44;&#x9636;&#x6BB5;&#x3001;&#x878D;&#x8D44;&#x91D1;&#x989D;&#x3001;&#x6295;&#x8D44;&#x516C;&#x53F8;</span>
    tz_info = scrapy.Field()
    <span class="hljs-comment"># &#x56E2;&#x961F;&#x4FE1;&#x606F;&#x5217;&#x8868;&#xFF1A;&#x5305;&#x542B;&#x6210;&#x5458;&#x59D3;&#x540D;&#x3001;&#x6210;&#x5458;&#x804C;&#x79F0;&#x3001;&#x6210;&#x5458;&#x4ECB;&#x7ECD;</span>
    tm_info = scrapy.Field()
    <span class="hljs-comment"># &#x4EA7;&#x54C1;&#x4FE1;&#x606F;&#x5217;&#x8868;&#xFF1A;&#x5305;&#x542B;&#x4EA7;&#x54C1;&#x540D;&#x79F0;&#x3001;&#x4EA7;&#x54C1;&#x7C7B;&#x578B;&#x3001;&#x4EA7;&#x54C1;&#x4ECB;&#x7ECD;</span>
    pdt_info = scrapy.Field()
</code></pre>
<h2 id="settingspy">settings.py</h2>
<pre><code class="lang-python">
<span class="hljs-comment"># -*- coding: utf-8 -*-</span>

BOT_NAME = <span class="hljs-string">&apos;itjuzi&apos;</span>

SPIDER_MODULES = [<span class="hljs-string">&apos;itjuzi.spiders&apos;</span>]
NEWSPIDER_MODULE = <span class="hljs-string">&apos;itjuzi.spiders&apos;</span>

<span class="hljs-comment"># Enables scheduling storing requests queue in redis.</span>
SCHEDULER = <span class="hljs-string">&quot;scrapy_redis.scheduler.Scheduler&quot;</span>

<span class="hljs-comment"># Ensure all spiders share same duplicates filter through redis.</span>
DUPEFILTER_CLASS = <span class="hljs-string">&quot;scrapy_redis.dupefilter.RFPDupeFilter&quot;</span>

<span class="hljs-comment"># REDIS_START_URLS_AS_SET = True</span>

COOKIES_ENABLED = <span class="hljs-keyword">False</span>

DOWNLOAD_DELAY = <span class="hljs-number">1.5</span>

<span class="hljs-comment"># &#x652F;&#x6301;&#x968F;&#x673A;&#x4E0B;&#x8F7D;&#x5EF6;&#x8FDF;</span>
RANDOMIZE_DOWNLOAD_DELAY = <span class="hljs-keyword">True</span>

<span class="hljs-comment"># Obey robots.txt rules</span>
ROBOTSTXT_OBEY = <span class="hljs-keyword">False</span>

ITEM_PIPELINES = {
    <span class="hljs-string">&apos;scrapy_redis.pipelines.RedisPipeline&apos;</span>: <span class="hljs-number">300</span>
}

DOWNLOADER_MIDDLEWARES = {
    <span class="hljs-comment"># &#x8BE5;&#x4E2D;&#x95F4;&#x4EF6;&#x5C06;&#x4F1A;&#x6536;&#x96C6;&#x5931;&#x8D25;&#x7684;&#x9875;&#x9762;&#xFF0C;&#x5E76;&#x5728;&#x722C;&#x866B;&#x5B8C;&#x6210;&#x540E;&#x91CD;&#x65B0;&#x8C03;&#x5EA6;&#x3002;&#xFF08;&#x5931;&#x8D25;&#x60C5;&#x51B5;&#x53EF;&#x80FD;&#x7531;&#x4E8E;&#x4E34;&#x65F6;&#x7684;&#x95EE;&#x9898;&#xFF0C;&#x4F8B;&#x5982;&#x8FDE;&#x63A5;&#x8D85;&#x65F6;&#x6216;&#x8005;HTTP 500&#x9519;&#x8BEF;&#x5BFC;&#x81F4;&#x5931;&#x8D25;&#x7684;&#x9875;&#x9762;&#xFF09;</span>
   <span class="hljs-string">&apos;scrapy.downloadermiddlewares.retry.RetryMiddleware&apos;</span>: <span class="hljs-number">80</span>,

    <span class="hljs-comment"># &#x8BE5;&#x4E2D;&#x95F4;&#x4EF6;&#x63D0;&#x4F9B;&#x4E86;&#x5BF9;request&#x8BBE;&#x7F6E;HTTP&#x4EE3;&#x7406;&#x7684;&#x652F;&#x6301;&#x3002;&#x60A8;&#x53EF;&#x4EE5;&#x901A;&#x8FC7;&#x5728; Request &#x5BF9;&#x8C61;&#x4E2D;&#x8BBE;&#x7F6E; proxy &#x5143;&#x6570;&#x636E;&#x6765;&#x5F00;&#x542F;&#x4EE3;&#x7406;&#x3002;</span>
    <span class="hljs-string">&apos;scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware&apos;</span>: <span class="hljs-number">100</span>,

    <span class="hljs-string">&apos;itjuzi.middlewares.RotateUserAgentMiddleware&apos;</span>: <span class="hljs-number">200</span>,
}

REDIS_HOST = <span class="hljs-string">&quot;192.168.199.108&quot;</span>
REDIS_PORT = <span class="hljs-number">6379</span>
</code></pre>
<h2 id="middlewarespy">middlewares.py</h2>
<pre><code class="lang-python"><span class="hljs-comment"># -*- coding: utf-8 -*-</span>

<span class="hljs-keyword">from</span> scrapy.contrib.downloadermiddleware.useragent <span class="hljs-keyword">import</span> UserAgentMiddleware
<span class="hljs-keyword">import</span> random

<span class="hljs-comment"># User-Agetn &#x4E0B;&#x8F7D;&#x4E2D;&#x95F4;&#x4EF6;</span>
<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">RotateUserAgentMiddleware</span><span class="hljs-params">(UserAgentMiddleware)</span>:</span>
    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self, user_agent=<span class="hljs-string">&apos;&apos;</span>)</span>:</span>
        self.user_agent = user_agent

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">process_request</span><span class="hljs-params">(self, request, spider)</span>:</span>
        <span class="hljs-comment"># &#x8FD9;&#x53E5;&#x8BDD;&#x7528;&#x4E8E;&#x968F;&#x673A;&#x9009;&#x62E9;user-agent</span>
        ua = random.choice(self.user_agent_list)
        request.headers.setdefault(<span class="hljs-string">&apos;User-Agent&apos;</span>, ua)

    user_agent_list = [
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.1.17) Gecko/20110123 (like Firefox/3.x) SeaMonkey/2.0.12&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Windows NT 5.2; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.54 Safari/535.2&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; U; Mac OS X Mach-O; en-US; rv:2.0a) Gecko/20040614 Firefox/3.0.0 &quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.0.3) Gecko/2008092414 Firefox/3.0.3&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.1) Gecko/20090624 Firefox/3.5&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.14) Gecko/20110218 AlexaToolbar/alxf-2.0 Firefox/3.6.14&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10.5; en-US; rv:1.9.2.15) Gecko/20110303 Firefox/3.6.15&quot;</span>,
        <span class="hljs-string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1&quot;</span>
    ]
</code></pre>
<h2 id="spidersjuzipy">spiders/juzi.py</h2>
<pre><code class="lang-python"><span class="hljs-comment"># -*- coding: utf-8 -*-</span>

<span class="hljs-keyword">from</span> bs4 <span class="hljs-keyword">import</span> BeautifulSoup
<span class="hljs-keyword">from</span> scrapy.linkextractors <span class="hljs-keyword">import</span> LinkExtractor
<span class="hljs-keyword">from</span> scrapy.spiders <span class="hljs-keyword">import</span> CrawlSpider, Rule

<span class="hljs-keyword">from</span> scrapy_redis.spiders <span class="hljs-keyword">import</span> RedisCrawlSpider
<span class="hljs-keyword">from</span> itjuzi.items <span class="hljs-keyword">import</span> CompanyItem


<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">ITjuziSpider</span><span class="hljs-params">(RedisCrawlSpider)</span>:</span>
    name = <span class="hljs-string">&apos;itjuzi&apos;</span>
    allowed_domains = [<span class="hljs-string">&apos;www.itjuzi.com&apos;</span>]
    <span class="hljs-comment"># start_urls = [&apos;http://www.itjuzi.com/company&apos;]</span>
    redis_key = <span class="hljs-string">&apos;itjuzispider:start_urls&apos;</span>
    rules = [
        <span class="hljs-comment"># &#x83B7;&#x53D6;&#x6BCF;&#x4E00;&#x9875;&#x7684;&#x94FE;&#x63A5;</span>
        Rule(link_extractor=LinkExtractor(allow=(<span class="hljs-string">&apos;/company\?page=\d+&apos;</span>))),
        <span class="hljs-comment"># &#x83B7;&#x53D6;&#x6BCF;&#x4E00;&#x4E2A;&#x516C;&#x53F8;&#x7684;&#x8BE6;&#x60C5;</span>
        Rule(link_extractor=LinkExtractor(allow=(<span class="hljs-string">&apos;/company/\d+&apos;</span>)), callback=<span class="hljs-string">&apos;parse_item&apos;</span>)
    ]

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">parse_item</span><span class="hljs-params">(self, response)</span>:</span>
        soup = BeautifulSoup(response.body, <span class="hljs-string">&apos;lxml&apos;</span>)

        <span class="hljs-comment"># &#x5F00;&#x5934;&#x90E8;&#x5206;&#xFF1A; //div[@class=&quot;infoheadrow-v2 ugc-block-item&quot;]</span>
        cpy1 = soup.find(<span class="hljs-string">&apos;div&apos;</span>, class_=<span class="hljs-string">&apos;infoheadrow-v2&apos;</span>)
        <span class="hljs-keyword">if</span> cpy1:
            <span class="hljs-comment"># &#x516C;&#x53F8;&#x540D;&#x79F0;&#xFF1A;//span[@class=&quot;title&quot;]/b/text()[1]</span>
            company_name = cpy1.find(class_=<span class="hljs-string">&apos;title&apos;</span>).b.contents[<span class="hljs-number">0</span>].strip().replace(<span class="hljs-string">&apos;\t&apos;</span>, <span class="hljs-string">&apos;&apos;</span>).replace(<span class="hljs-string">&apos;\n&apos;</span>, <span class="hljs-string">&apos;&apos;</span>)

            <span class="hljs-comment"># &#x53E3;&#x53F7;&#xFF1A; //div[@class=&quot;info-line&quot;]/p</span>
            slogan = cpy1.find(class_=<span class="hljs-string">&apos;info-line&apos;</span>).p.get_text()

            <span class="hljs-comment"># &#x5206;&#x7C7B;&#xFF1A;&#x5B50;&#x5206;&#x7C7B;//span[@class=&quot;scope c-gray-aset&quot;]/a[1]</span>
            scope_a = cpy1.find(class_=<span class="hljs-string">&apos;scope c-gray-aset&apos;</span>).find_all(<span class="hljs-string">&apos;a&apos;</span>)
            <span class="hljs-comment"># &#x5206;&#x7C7B;&#xFF1A;//span[@class=&quot;scope c-gray-aset&quot;]/a[1]</span>
            scope = scope_a[<span class="hljs-number">0</span>].get_text().strip() <span class="hljs-keyword">if</span> len(scope_a) &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&apos;&apos;</span>
            <span class="hljs-comment"># &#x5B50;&#x5206;&#x7C7B;&#xFF1A;# //span[@class=&quot;scope c-gray-aset&quot;]/a[2]</span>
            sub_scope = scope_a[<span class="hljs-number">1</span>].get_text().strip() <span class="hljs-keyword">if</span> len(scope_a) &gt; <span class="hljs-number">1</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&apos;&apos;</span>

            <span class="hljs-comment"># &#x57CE;&#x5E02;+&#x533A;&#x57DF;&#xFF1A;//span[@class=&quot;loca c-gray-aset&quot;]/a</span>
            city_a = cpy1.find(class_=<span class="hljs-string">&apos;loca c-gray-aset&apos;</span>).find_all(<span class="hljs-string">&apos;a&apos;</span>)
            <span class="hljs-comment"># &#x57CE;&#x5E02;&#xFF1A;//span[@class=&quot;loca c-gray-aset&quot;]/a[1]</span>
            city = city_a[<span class="hljs-number">0</span>].get_text().strip() <span class="hljs-keyword">if</span> len(city_a) &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&apos;&apos;</span>
            <span class="hljs-comment"># &#x533A;&#x57DF;&#xFF1A;//span[@class=&quot;loca c-gray-aset&quot;]/a[2]</span>
            area = city_a[<span class="hljs-number">1</span>].get_text().strip() <span class="hljs-keyword">if</span> len(city_a) &gt; <span class="hljs-number">1</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&apos;&apos;</span>

            <span class="hljs-comment"># &#x4E3B;&#x9875;&#xFF1A;//a[@class=&quot;weblink marl10&quot;]/@href</span>
            home_page = cpy1.find(class_=<span class="hljs-string">&apos;weblink marl10&apos;</span>)[<span class="hljs-string">&apos;href&apos;</span>]
            <span class="hljs-comment"># &#x6807;&#x7B7E;&#xFF1A;//div[@class=&quot;tagset dbi c-gray-aset&quot;]/a</span>
            tags = cpy1.find(class_=<span class="hljs-string">&apos;tagset dbi c-gray-aset&apos;</span>).get_text().strip().strip().replace(<span class="hljs-string">&apos;\n&apos;</span>, <span class="hljs-string">&apos;,&apos;</span>)

        <span class="hljs-comment">#&#x57FA;&#x672C;&#x4FE1;&#x606F;&#xFF1A;//div[@class=&quot;block-inc-info on-edit-hide&quot;]</span>
        cpy2 = soup.find(<span class="hljs-string">&apos;div&apos;</span>, class_=<span class="hljs-string">&apos;block-inc-info on-edit-hide&apos;</span>)
        <span class="hljs-keyword">if</span> cpy2:

            <span class="hljs-comment"># &#x516C;&#x53F8;&#x7B80;&#x4ECB;&#xFF1A;//div[@class=&quot;block-inc-info on-edit-hide&quot;]//div[@class=&quot;des&quot;]</span>
            company_intro = cpy2.find(class_=<span class="hljs-string">&apos;des&apos;</span>).get_text().strip()

            <span class="hljs-comment"># &#x516C;&#x53F8;&#x5168;&#x79F0;&#xFF1A;&#x6210;&#x7ACB;&#x65F6;&#x95F4;&#xFF1A;&#x516C;&#x53F8;&#x89C4;&#x6A21;&#xFF1A;&#x8FD0;&#x884C;&#x72B6;&#x6001;&#xFF1A;//div[@class=&quot;des-more&quot;]</span>
            cpy2_content = cpy2.find(class_=<span class="hljs-string">&apos;des-more&apos;</span>).contents

            <span class="hljs-comment"># &#x516C;&#x53F8;&#x5168;&#x79F0;&#xFF1A;//div[@class=&quot;des-more&quot;]/div[1]</span>
            company_full_name = cpy2_content[<span class="hljs-number">1</span>].get_text().strip()[len(<span class="hljs-string">&apos;&#x516C;&#x53F8;&#x5168;&#x79F0;&#xFF1A;&apos;</span>):] <span class="hljs-keyword">if</span> cpy2_content[<span class="hljs-number">1</span>] <span class="hljs-keyword">else</span> <span class="hljs-string">&apos;&apos;</span>

            <span class="hljs-comment"># &#x6210;&#x7ACB;&#x65F6;&#x95F4;&#xFF1A;//div[@class=&quot;des-more&quot;]/div[2]/span[1]</span>
            found_time = cpy2_content[<span class="hljs-number">3</span>].contents[<span class="hljs-number">1</span>].get_text().strip()[len(<span class="hljs-string">&apos;&#x6210;&#x7ACB;&#x65F6;&#x95F4;&#xFF1A;&apos;</span>):] <span class="hljs-keyword">if</span> cpy2_content[<span class="hljs-number">3</span>] <span class="hljs-keyword">else</span> <span class="hljs-string">&apos;&apos;</span>

            <span class="hljs-comment"># &#x516C;&#x53F8;&#x89C4;&#x6A21;&#xFF1A;//div[@class=&quot;des-more&quot;]/div[2]/span[2]</span>
            company_size = cpy2_content[<span class="hljs-number">3</span>].contents[<span class="hljs-number">3</span>].get_text().strip()[len(<span class="hljs-string">&apos;&#x516C;&#x53F8;&#x89C4;&#x6A21;&#xFF1A;&apos;</span>):] <span class="hljs-keyword">if</span> cpy2_content[<span class="hljs-number">3</span>] <span class="hljs-keyword">else</span> <span class="hljs-string">&apos;&apos;</span>

            <span class="hljs-comment">#&#x8FD0;&#x8425;&#x72B6;&#x6001;&#xFF1A;//div[@class=&quot;des-more&quot;]/div[3]</span>
            company_status = cpy2_content[<span class="hljs-number">5</span>].get_text().strip() <span class="hljs-keyword">if</span> cpy2_content[<span class="hljs-number">5</span>] <span class="hljs-keyword">else</span> <span class="hljs-string">&apos;&apos;</span>

        <span class="hljs-comment"># &#x4E3B;&#x4F53;&#x4FE1;&#x606F;&#xFF1A;</span>
        main = soup.find(<span class="hljs-string">&apos;div&apos;</span>, class_=<span class="hljs-string">&apos;main&apos;</span>)

        <span class="hljs-comment"># &#x6295;&#x8D44;&#x60C5;&#x51B5;&#xFF1A;//table[@class=&quot;list-round-v2 need2login&quot;]</span>
          <span class="hljs-comment"># &#x6295;&#x8D44;&#x60C5;&#x51B5;&#xFF0C;&#x5305;&#x542B;&#x83B7;&#x6295;&#x65F6;&#x95F4;&#x3001;&#x878D;&#x8D44;&#x9636;&#x6BB5;&#x3001;&#x878D;&#x8D44;&#x91D1;&#x989D;&#x3001;&#x6295;&#x8D44;&#x516C;&#x53F8;</span>
        tz = main.find(<span class="hljs-string">&apos;table&apos;</span>, <span class="hljs-string">&apos;list-round-v2&apos;</span>)
        tz_list = []
        <span class="hljs-keyword">if</span> tz:
            all_tr = tz.find_all(<span class="hljs-string">&apos;tr&apos;</span>)
            <span class="hljs-keyword">for</span> tr <span class="hljs-keyword">in</span> all_tr:
                tz_dict = {}
                all_td = tr.find_all(<span class="hljs-string">&apos;td&apos;</span>)
                tz_dict[<span class="hljs-string">&apos;tz_time&apos;</span>] = all_td[<span class="hljs-number">0</span>].span.get_text().strip()
                tz_dict[<span class="hljs-string">&apos;tz_round&apos;</span>] = all_td[<span class="hljs-number">1</span>].get_text().strip()
                tz_dict[<span class="hljs-string">&apos;tz_finades&apos;</span>] = all_td[<span class="hljs-number">2</span>].get_text().strip()
                tz_dict[<span class="hljs-string">&apos;tz_capital&apos;</span>] = all_td[<span class="hljs-number">3</span>].get_text().strip().replace(<span class="hljs-string">&apos;\n&apos;</span>, <span class="hljs-string">&apos;,&apos;</span>)
                tz_list.append(tz_dict)

        <span class="hljs-comment"># &#x56E2;&#x961F;&#x4FE1;&#x606F;&#xFF1A;&#x6210;&#x5458;&#x59D3;&#x540D;&#x3001;&#x6210;&#x5458;&#x804C;&#x79F0;&#x3001;&#x6210;&#x5458;&#x4ECB;&#x7ECD;</span>
        tm = main.find(<span class="hljs-string">&apos;ul&apos;</span>, class_=<span class="hljs-string">&apos;list-prodcase limited-itemnum&apos;</span>)
        tm_list = []
        <span class="hljs-keyword">if</span> tm:
            <span class="hljs-keyword">for</span> li <span class="hljs-keyword">in</span> tm.find_all(<span class="hljs-string">&apos;li&apos;</span>):
                tm_dict = {}
                tm_dict[<span class="hljs-string">&apos;tm_m_name&apos;</span>] = li.find(<span class="hljs-string">&apos;span&apos;</span>, class_=<span class="hljs-string">&apos;c&apos;</span>).get_text().strip()
                tm_dict[<span class="hljs-string">&apos;tm_m_title&apos;</span>] = li.find(<span class="hljs-string">&apos;span&apos;</span>, class_=<span class="hljs-string">&apos;c-gray&apos;</span>).get_text().strip()
                tm_dict[<span class="hljs-string">&apos;tm_m_intro&apos;</span>] = li.find(<span class="hljs-string">&apos;p&apos;</span>, class_=<span class="hljs-string">&apos;mart10 person-des&apos;</span>).get_text().strip()
                tm_list.append(tm_dict)

        <span class="hljs-comment"># &#x4EA7;&#x54C1;&#x4FE1;&#x606F;&#xFF1A;&#x4EA7;&#x54C1;&#x540D;&#x79F0;&#x3001;&#x4EA7;&#x54C1;&#x7C7B;&#x578B;&#x3001;&#x4EA7;&#x54C1;&#x4ECB;&#x7ECD;</span>
        pdt = main.find(<span class="hljs-string">&apos;ul&apos;</span>, class_=<span class="hljs-string">&apos;list-prod limited-itemnum&apos;</span>)
        pdt_list = []
        <span class="hljs-keyword">if</span> pdt:
            <span class="hljs-keyword">for</span> li <span class="hljs-keyword">in</span> pdt.find_all(<span class="hljs-string">&apos;li&apos;</span>):
                pdt_dict = {}
                pdt_dict[<span class="hljs-string">&apos;pdt_name&apos;</span>] = li.find(<span class="hljs-string">&apos;h4&apos;</span>).b.get_text().strip()
                pdt_dict[<span class="hljs-string">&apos;pdt_type&apos;</span>] = li.find(<span class="hljs-string">&apos;span&apos;</span>, class_=<span class="hljs-string">&apos;tag yellow&apos;</span>).get_text().strip()
                pdt_dict[<span class="hljs-string">&apos;pdt_intro&apos;</span>] = li.find(class_=<span class="hljs-string">&apos;on-edit-hide&apos;</span>).p.get_text().strip()
                pdt_list.append(pdt_dict)

        item = CompanyItem()
        item[<span class="hljs-string">&apos;info_id&apos;</span>] = response.url.split(<span class="hljs-string">&apos;/&apos;</span>)[-<span class="hljs-number">1</span>:][<span class="hljs-number">0</span>]
        item[<span class="hljs-string">&apos;company_name&apos;</span>] = company_name
        item[<span class="hljs-string">&apos;slogan&apos;</span>] = slogan
        item[<span class="hljs-string">&apos;scope&apos;</span>] = scope
        item[<span class="hljs-string">&apos;sub_scope&apos;</span>] = sub_scope
        item[<span class="hljs-string">&apos;city&apos;</span>] = city
        item[<span class="hljs-string">&apos;area&apos;</span>] = area
        item[<span class="hljs-string">&apos;home_page&apos;</span>] = home_page
        item[<span class="hljs-string">&apos;tags&apos;</span>] = tags
        item[<span class="hljs-string">&apos;company_intro&apos;</span>] = company_intro
        item[<span class="hljs-string">&apos;company_full_name&apos;</span>] = company_full_name
        item[<span class="hljs-string">&apos;found_time&apos;</span>] = found_time
        item[<span class="hljs-string">&apos;company_size&apos;</span>] = company_size
        item[<span class="hljs-string">&apos;company_status&apos;</span>] = company_status
        item[<span class="hljs-string">&apos;tz_info&apos;</span>] = tz_list
        item[<span class="hljs-string">&apos;tm_info&apos;</span>] = tm_list
        item[<span class="hljs-string">&apos;pdt_info&apos;</span>] = pdt_list
        <span class="hljs-keyword">return</span> item
</code></pre>
<h2 id="scrapycfg">scrapy.cfg</h2>
<pre><code class="lang-python"><span class="hljs-comment"># Automatically created by: scrapy startproject</span>
<span class="hljs-comment">#</span>
<span class="hljs-comment"># For more information about the [deploy] section see:</span>
<span class="hljs-comment"># https://scrapyd.readthedocs.org/en/latest/deploy.html</span>

[settings]
default = itjuzi.settings

[deploy]
<span class="hljs-comment">#url = http://localhost:6800/</span>
project = itjuzi
</code></pre>
<h2 id="&#x8FD0;&#x884C;&#xFF1A;">&#x8FD0;&#x884C;&#xFF1A;</h2>
<pre><code class="lang-python">Slave&#x7AEF;&#xFF1A;
scrapy runspider juzi.py

Master&#x7AEF;&#xFF1A;
redis-cli &gt; lpush itjuzispider:start_urls http://www.itjuzi.com/company
</code></pre>
<h2 id="&#x6F14;&#x793A;&#x6548;&#x679C;&#xFF1A;">&#x6F14;&#x793A;&#x6548;&#x679C;&#xFF1A;</h2>
<p><img src="../images/itjuzi_0.png" alt=""></p>
<p><img src="../images/itjuzi_1.png" alt=""> </p>
<footer class="page-footer"><span class="copyright">Copyright &#xA9; BigCat all right reserved&#xFF0C;powered by Gitbook</span><span class="footer-modification">&#x300C;Revision Time:
2017-02-09 01:28:05&#x300D;
</span></footer>
                    
                    </section>
                
                
                </div>
            </div>
        </div>

        
        <a href="../../file/part07/7.7.html" class="navigation navigation-prev " aria-label="Previous page: IT桔子分布式项目1"><i class="fa fa-angle-left"></i></a>
        
        
        <a href="../../file/duanzi/duanzi.html" class="navigation navigation-next " aria-label="Next page: 课余段子"><i class="fa fa-angle-right"></i></a>
        
    </div>
</div>

        
<script src="../../gitbook/app.js"></script>

    
    <script src="../../gitbook/plugins/gitbook-plugin-splitter/splitter.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-toggle-chapters/toggle.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-fontsettings/buttons.js"></script>
    

    
    <script src="../../gitbook/plugins/gitbook-plugin-livereload/plugin.js"></script>
    

<script>
require(["gitbook"], function(gitbook) {
    var config = {"disqus":{"shortName":"gitbookuse"},"github":{"url":"https://github.com/dododream"},"search-pro":{"cutWordLib":"nodejieba","defineWord":["gitbook-use"]},"sharing":{"weibo":true,"facebook":true,"twitter":true,"google":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"tbfed-pagefooter":{"copyright":"Copyright © BigCat","modify_label":"「Revision Time:","modify_format":"YYYY-MM-DD HH:mm:ss」"},"baidu":{"token":"ff100361cdce95dd4c8fb96b4009f7bc"},"sitemap":{"hostname":"http://www.treenewbee.top"},"donate":{"wechat":"http://weixin.png","alipay":"http://alipay.png","title":"","button":"赏","alipayText":"支付宝打赏","wechatText":"微信打赏"},"edit-link":{"base":"https://github.com/dododream/edit","label":"Edit This Page"},"splitter":{},"toggle-chapters":{},"highlight":{},"fontsettings":{"theme":"white","family":"sans","size":2},"livereload":{}};
    gitbook.start(config);
});
</script>

        <!-- body:end -->
    </body>
    <!-- End of book Python爬虫课程讲义 -->
</html>
